{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from hfnet.datasets.sfm import Sfm\n",
    "from hfnet.evaluation.loaders import sift_loader, export_loader, fast_loader, harris_loader\n",
    "from hfnet.evaluation.keypoint_detectors import evaluate\n",
    "from hfnet.utils import tools\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01/10/2019 15:08:49 INFO] SfM sequence british_museum contains 50 pairs\n",
      "[01/10/2019 15:08:51 INFO] SfM sequence florence_cathedral_side contains 50 pairs\n",
      "[01/10/2019 15:09:03 INFO] SfM sequence lincoln_memorial_statue contains 50 pairs\n",
      "[01/10/2019 15:09:12 INFO] SfM sequence london_bridge contains 50 pairs\n",
      "[01/10/2019 15:09:13 INFO] SfM sequence milan_cathedral contains 50 pairs\n",
      "[01/10/2019 15:09:15 INFO] SfM sequence mount_rushmore contains 50 pairs\n",
      "[01/10/2019 15:09:19 INFO] SfM sequence piazza_san_marco contains 50 pairs\n",
      "[01/10/2019 15:09:19 INFO] No EXIF data for sequence reichstag\n",
      "[01/10/2019 15:09:19 INFO] SfM sequence reichstag contains 50 pairs\n",
      "[01/10/2019 15:09:26 INFO] SfM sequence sagrada_familia contains 50 pairs\n",
      "[01/10/2019 15:09:38 INFO] SfM sequence st_pauls_cathedral contains 50 pairs\n",
      "[01/10/2019 15:09:45 INFO] SfM sequence united_states_capitol contains 50 pairs\n"
     ]
    }
   ],
   "source": [
    "config = {\n",
    "    'sequences': ['british_museum', 'florence_cathedral_side', 'lincoln_memorial_statue', \n",
    "                  'london_bridge', 'milan_cathedral', 'mount_rushmore', 'piazza_san_marco',\n",
    "                  'reichstag', 'sagrada_familia', 'st_pauls_cathedral', 'united_states_capitol'],\n",
    "    'make_pairs': True, 'shuffle': True, 'pair_thresh': 0.4, 'max_num_pairs': 50,\n",
    "    'preprocessing': {'upright': True},\n",
    "}\n",
    "dataset = Sfm(**config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_configs = {\n",
    "    'sift': {\n",
    "        'predictor': sift_loader,\n",
    "        'do_nms': False,\n",
    "        'nms_thresh': 8,\n",
    "    },\n",
    "    'harris': {\n",
    "        'predictor': harris_loader,\n",
    "        'do_nms': True,\n",
    "        'nms_thresh': 4,\n",
    "    },\n",
    "    'fast': {\n",
    "        'predictor': fast_loader,\n",
    "        'do_nms': True,\n",
    "        'nms_thresh': 4,\n",
    "    },\n",
    "    'superpoint': {\n",
    "        'experiment': 'super_point_pytorch/sfm_upright',\n",
    "        'predictor': export_loader,\n",
    "        'do_nms': True,\n",
    "        'nms_thresh': 4,\n",
    "        'remove_borders': 4,\n",
    "        'has_descriptors': False\n",
    "    },\n",
    "    'lfnet': {\n",
    "        'experiment': 'lfnet/sfm_upright_kpts-1000',\n",
    "        'predictor': export_loader,\n",
    "        'do_nms': False,\n",
    "        'nms_thresh': 4,\n",
    "    },\n",
    "}\n",
    "eval_config = {'correct_match_thresh': 3, 'num_features': 1000}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "methods = ['sift', 'harris', 'fast', 'superpoint', 'lfnet']\n",
    "configs = {m: all_configs[m] for m in methods}\n",
    "for method, config in configs.items():\n",
    "    config = tools.dict_update(config, eval_config)\n",
    "    data_iter = dataset.get_test_set()\n",
    "    metrics, _, _, _ = evaluate(data_iter, config, is_2d=False)\n",
    "    \n",
    "    print('> {}'.format(method))\n",
    "    for k, v in metrics.items():\n",
    "        print('{:<25} {:.3f}'.format(k, v))\n",
    "    print(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "550it [03:38,  2.85it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sift\n",
      "average_num_keypoints     1000.000\n",
      "localization_error        1.197\n",
      "repeatability             0.284\n",
      "mAP                       0.092\n",
      "{'predictor': <function sift_loader at 0x2abc48e76e18>, 'do_nms': False, 'nms_thresh': 8, 'correct_match_thresh': 3, 'num_features': 1000}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "550it [03:22,  3.15it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> harris\n",
      "average_num_keypoints     980.197\n",
      "localization_error        1.458\n",
      "repeatability             0.510\n",
      "mAP                       0.308\n",
      "{'predictor': <function harris_loader at 0x2abc48e959d8>, 'do_nms': True, 'nms_thresh': 4, 'correct_match_thresh': 3, 'num_features': 1000}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "550it [01:47,  6.73it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> fast\n",
      "average_num_keypoints     975.471\n",
      "localization_error        1.419\n",
      "repeatability             0.451\n",
      "mAP                       0.249\n",
      "{'predictor': <function fast_loader at 0x2abc48e95950>, 'do_nms': True, 'nms_thresh': 4, 'correct_match_thresh': 3, 'num_features': 1000}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "550it [01:34,  5.98it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> superpoint\n",
      "average_num_keypoints     828.831\n",
      "localization_error        1.451\n",
      "repeatability             0.509\n",
      "mAP                       0.329\n",
      "{'experiment': 'super_point_pytorch/colmap_upright', 'predictor': <function export_loader at 0x2abc48e95a60>, 'do_nms': True, 'nms_thresh': 4, 'remove_borders': 4, 'has_descriptors': False, 'correct_match_thresh': 3, 'num_features': 1000}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "550it [00:59,  9.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> lfnet\n",
      "average_num_keypoints     1000.000\n",
      "localization_error        1.439\n",
      "repeatability             0.454\n",
      "mAP                       0.258\n",
      "{'experiment': 'lfnet/colmap_upright_kpts-1000', 'predictor': <function export_loader at 0x2abc48e95a60>, 'do_nms': False, 'nms_thresh': 4, 'correct_match_thresh': 3, 'num_features': 1000}\n"
     ]
    }
   ],
   "source": [
    "# NMS=4, N=1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "550it [03:58,  2.72it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sift\n",
      "average_num_keypoints     1000.000\n",
      "localization_error        1.197\n",
      "repeatability             0.284\n",
      "mAP                       0.092\n",
      "{'predictor': <function sift_loader at 0x2abc48e76e18>, 'do_nms': False, 'nms_thresh': 8, 'correct_match_thresh': 3, 'num_features': 1000}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "550it [03:05,  3.72it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> harris\n",
      "average_num_keypoints     878.683\n",
      "localization_error        1.507\n",
      "repeatability             0.427\n",
      "mAP                       0.221\n",
      "{'predictor': <function harris_loader at 0x2abc48e959d8>, 'do_nms': True, 'nms_thresh': 8, 'correct_match_thresh': 3, 'num_features': 1000}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "550it [01:27,  8.20it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> fast\n",
      "average_num_keypoints     860.036\n",
      "localization_error        1.441\n",
      "repeatability             0.337\n",
      "mAP                       0.147\n",
      "{'predictor': <function fast_loader at 0x2abc48e95950>, 'do_nms': True, 'nms_thresh': 8, 'correct_match_thresh': 3, 'num_features': 1000}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "550it [01:48,  5.06it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> superpoint\n",
      "average_num_keypoints     686.645\n",
      "localization_error        1.455\n",
      "repeatability             0.476\n",
      "mAP                       0.283\n",
      "{'experiment': 'super_point_pytorch/colmap_upright', 'predictor': <function export_loader at 0x2abc48e95a60>, 'do_nms': True, 'nms_thresh': 8, 'remove_borders': 4, 'has_descriptors': False, 'correct_match_thresh': 3, 'num_features': 1000}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "550it [01:24,  9.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> lfnet\n",
      "average_num_keypoints     1000.000\n",
      "localization_error        1.439\n",
      "repeatability             0.454\n",
      "mAP                       0.258\n",
      "{'experiment': 'lfnet/colmap_upright_kpts-1000', 'predictor': <function export_loader at 0x2abc48e95a60>, 'do_nms': False, 'nms_thresh': 8, 'correct_match_thresh': 3, 'num_features': 1000}\n"
     ]
    }
   ],
   "source": [
    "# NMS=8, N=1000"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
